{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 模型结构\n",
    "\n",
    "- 两层单向LSTM, 输出序列结果, 即(batch_size, step_size, feature_size)\n",
    "- 分别输入到1, 2, 3, 4, 5, 6共6个不同长度的卷积层中\n",
    "- 卷积层为双层, 最后的池化层有Average和Max两种\n",
    "- 对于每个问题, 将所有卷积核结果并起来\n",
    "- 将两个问题并起来的结果, 分别[相减并取绝对值], [x]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import warnings\n",
    "warnings.filterwarnings(\"ignore\")\n",
    "\n",
    "import os\n",
    "import shutil\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "from glob import glob\n",
    "from datetime import datetime"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "try:\n",
    "    os.mkdir(\"./log/\")\n",
    "    os.mkdir(\"./result/\")\n",
    "except FileExistsError:\n",
    "    pass"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "DATA_PATH = \"./data/\"\n",
    "TRAIN_PATH = DATA_PATH + \"train.csv\"\n",
    "TEST_PATH = DATA_PATH + \"test.csv\"\n",
    "WORD_EMBED_PATH = DATA_PATH + \"word_embed.txt\"\n",
    "CHAR_EMBED_PATH = DATA_PATH + \"char_embed.txt\"\n",
    "QUEST_PATH = DATA_PATH + \"question.csv\"\n",
    "\n",
    "train_data = pd.read_csv(TRAIN_PATH)\n",
    "test_data = pd.read_csv(TEST_PATH)\n",
    "question_data = pd.read_csv(QUEST_PATH)\n",
    "word_embedding_data = pd.read_csv(WORD_EMBED_PATH, delimiter=\" \", header=None, index_col=0)\n",
    "char_embedding_data = pd.read_csv(CHAR_EMBED_PATH, delimiter=\" \", header=None, index_col=0)\n",
    "\n",
    "question_data[\"words\"] = question_data[\"words\"].str.split(\" \")\n",
    "question_data[\"chars\"] = question_data[\"chars\"].str.split(\" \")\n",
    "\n",
    "label = train_data[\"label\"].values\n",
    "\n",
    "from keras.preprocessing.text import Tokenizer\n",
    "\n",
    "MAX_COUNT = 10000\n",
    "\n",
    "word_tokenizer = Tokenizer(MAX_COUNT)\n",
    "word_tokenizer.fit_on_texts(question_data[\"words\"])\n",
    "\n",
    "word_embedding_data = np.concatenate(\n",
    "    (\n",
    "        np.zeros(shape=(1, word_embedding_data.shape[1]), dtype=np.float64),\n",
    "        word_embedding_data.loc[list(word_tokenizer.word_index.keys())[:MAX_COUNT]].values\n",
    "    ),\n",
    "    axis=0\n",
    ")\n",
    "\n",
    "char_tokenizer = Tokenizer(MAX_COUNT)\n",
    "char_tokenizer.fit_on_texts(question_data[\"chars\"])\n",
    "\n",
    "char_embedding_data = np.concatenate(\n",
    "    (\n",
    "        np.zeros(shape=(1, char_embedding_data.shape[1]), dtype=np.float64),\n",
    "        char_embedding_data.loc[list(char_tokenizer.word_index.keys())[:MAX_COUNT]].values\n",
    "    ),\n",
    "    axis=0\n",
    ")\n",
    "\n",
    "word_embedding_data.shape, char_embedding_data.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from keras.preprocessing.sequence import pad_sequences\n",
    "\n",
    "SEQ_LEN = 25\n",
    "\n",
    "def gen_word_data(data):\n",
    "    seq_word1 = word_tokenizer.texts_to_sequences(data.merge(question_data, how=\"left\", left_on=\"q1\", right_on=\"qid\")[\"words\"])\n",
    "    seq_word2 = word_tokenizer.texts_to_sequences(data.merge(question_data, how=\"left\", left_on=\"q2\", right_on=\"qid\")[\"words\"])\n",
    "    return pad_sequences(seq_word1, maxlen=SEQ_LEN, padding=\"pre\",truncating=\"pre\"), \\\n",
    "        pad_sequences(seq_word2, maxlen=SEQ_LEN, padding=\"pre\",truncating=\"pre\")\n",
    "    \n",
    "def gen_char_data(data):\n",
    "    seq_char1 = char_tokenizer.texts_to_sequences(data.merge(question_data, how=\"left\", left_on=\"q1\", right_on=\"qid\")[\"chars\"])\n",
    "    seq_char2 = char_tokenizer.texts_to_sequences(data.merge(question_data, how=\"left\", left_on=\"q2\", right_on=\"qid\")[\"chars\"])\n",
    "    return pad_sequences(seq_char1, maxlen=SEQ_LEN, padding=\"pre\",truncating=\"pre\"), \\\n",
    "        pad_sequences(seq_char2, maxlen=SEQ_LEN, padding=\"pre\",truncating=\"pre\")\n",
    "\n",
    "word1, word2 = gen_word_data(train_data)\n",
    "char1, char2 = gen_char_data(train_data)\n",
    "test_word1, test_word2 = gen_word_data(test_data)\n",
    "test_char1, test_char2 = gen_char_data(test_data)\n",
    "\n",
    "word1.shape, word2.shape, test_word1.shape, test_word2.shape, char1.shape, char2.shape, test_char1.shape, test_char2.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from keras.models import Model\n",
    "from keras.layers.merge import concatenate\n",
    "from keras.optimizers import Adam, SGD, Nadam\n",
    "from keras.callbacks import EarlyStopping, ModelCheckpoint, ReduceLROnPlateau\n",
    "from keras.layers import LSTM, Bidirectional, TimeDistributed\n",
    "from keras.layers import Conv1D, MaxPool1D, GlobalAveragePooling1D, GlobalMaxPooling1D\n",
    "from keras.layers import Input, Embedding, Dropout, BatchNormalization, Dense, Flatten, Lambda, K, Activation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# general\n",
    "NUM_EPOCHES = 50\n",
    "EPOCHES1 = 5\n",
    "EPOCHES2 = 25 # 5\n",
    "EPOCHES3 = 22\n",
    "BATCH_SIZE = 1024\n",
    "DROP_RATE = 0.3\n",
    "\n",
    "# cnn\n",
    "CONV_LEN1 = 128\n",
    "CONV_LEN2 = 128\n",
    "CONV_LEN3 = 128\n",
    "CONV_LEN4 = 128\n",
    "CONV_LEN5 = 128\n",
    "CONV_LEN6 = 128\n",
    "\n",
    "# lstm\n",
    "LSTM_SIZE1 = 256\n",
    "LSTM_SIZE2 = 256\n",
    "LSTM_DROP_RATE = 0.3\n",
    "\n",
    "# dense\n",
    "DENSE_INPUT = 300\n",
    "DENSE_SIZE1 = 512\n",
    "DENSE_SIZE2 = 256"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def cnn_layer1(inputa, inputb, filters, kernel_size): # with average pooling\n",
    "    conv = Conv1D(filters=filters, kernel_size=kernel_size, padding=\"same\", activation=\"relu\")\n",
    "    conv_outputa = conv(inputa)\n",
    "    conv_outputa = GlobalAveragePooling1D()(conv_outputa)\n",
    "    conv_outputb = conv(inputb)\n",
    "    conv_outputb = GlobalAveragePooling1D()(conv_outputb)\n",
    "    return conv_outputa, conv_outputb\n",
    "    \n",
    "def cnn_layer2(inputa, inputb, filters, kernel_size): # with max pooling\n",
    "    conv = Conv1D(filters=filters, kernel_size=kernel_size, padding=\"same\", activation=\"relu\")\n",
    "    conv_outputa = conv(inputa)\n",
    "    conv_outputa = MaxPool1D(pool_size=SEQ_LEN)(conv_outputa)\n",
    "    conv_outputa = Flatten()(conv_outputa)\n",
    "    conv_outputb = conv(inputb)\n",
    "    conv_outputb = MaxPool1D(pool_size=SEQ_LEN)(conv_outputb)\n",
    "    conv_outputb = Flatten()(conv_outputb)\n",
    "    return conv_outputa, conv_outputb\n",
    "\n",
    "def cnn_layer3(inputa, inputb, filters, kernel_size): # with both max and average poolings\n",
    "    conv = Conv1D(filters=filters, kernel_size=kernel_size, padding=\"same\", activation=\"relu\")\n",
    "    \n",
    "    conv_outputa = conv(inputa)\n",
    "    conv_outputa1 = Flatten()(MaxPool1D(pool_size=SEQ_LEN)(conv_outputa))\n",
    "    conv_outputa2 = GlobalAveragePooling1D()(conv_outputa)\n",
    "    conv_outputa = concatenate([conv_outputa1, conv_outputa2])\n",
    "    \n",
    "    conv_outputb = conv(inputb)\n",
    "    conv_outputb1 = Flatten()(MaxPool1D(pool_size=SEQ_LEN)(conv_outputb))\n",
    "    conv_outputb2 = GlobalAveragePooling1D()(conv_outputb)\n",
    "    conv_outputb = concatenate([conv_outputb1, conv_outputb2])\n",
    "    \n",
    "    return conv_outputa, conv_outputb\n",
    "\n",
    "def cnn_double_layer(inputa, inputb, filters, kernel_size):\n",
    "    conv1 = Conv1D(filters=filters, kernel_size=kernel_size, padding=\"same\")\n",
    "    conv2 = Conv1D(filters=filters, kernel_size=kernel_size, padding=\"same\")\n",
    "    \n",
    "    conv1a = conv1(inputa)\n",
    "    conv1a = BatchNormalization()(conv1a)\n",
    "    conv1a = Activation(activation=\"relu\")(conv1a)\n",
    "    conv2a = conv2(conv1a)\n",
    "    conv2a = BatchNormalization()(conv2a)\n",
    "    conv2a = Activation(activation=\"relu\")(conv2a)\n",
    "    output_avg_a = GlobalAveragePooling1D()(conv2a)\n",
    "    output_max_a = Flatten()(MaxPool1D(pool_size=SEQ_LEN)(conv2a))\n",
    "    output_min_a = Lambda(lambda x: K.min(x, axis=1))(conv2a)\n",
    "    output_a = concatenate([output_avg_a, output_max_a, output_min_a])\n",
    "    \n",
    "    conv1b = conv1(inputb)\n",
    "    conv1b = BatchNormalization()(conv1b)\n",
    "    conv1b = Activation(activation=\"relu\")(conv1b)\n",
    "    conv2b = conv2(conv1b)\n",
    "    conv2b = BatchNormalization()(conv2b)\n",
    "    conv2b = Activation(activation=\"relu\")(conv2b)\n",
    "    output_avg_b = GlobalAveragePooling1D()(conv2b)\n",
    "    output_max_b = Flatten()(MaxPool1D(pool_size=SEQ_LEN)(conv2b))\n",
    "    output_min_b = Lambda(lambda x: K.min(x, axis=1))(conv2b)\n",
    "    output_b = concatenate([output_avg_b, output_max_b, output_min_b])\n",
    "    \n",
    "    return output_a, output_b\n",
    "\n",
    "\n",
    "def sim_l1(v1, v2):\n",
    "    return Lambda(lambda x: K.sum(K.abs(x[0] - x[1]), axis=1))([v1, v2])\n",
    "\n",
    "def sim_l2(v1, v2):\n",
    "    return Lambda(lambda x: K.sqrt(K.sum(K.square(x[0] - x[1]), axis=1)))([v1, v2])\n",
    "\n",
    "def sim_cos(v1, v2):\n",
    "    return Lambda(lambda x: K.sum(x[0] * x[1], axis=1) / (K.sqrt(K.sum(x[0] * x[0], axis=1)) * K.sqrt(K.sum(x[1] * x[1], axis=1))))([v1, v2])\n",
    "\n",
    "def sim_vec(v1, v2):\n",
    "    l1 = sim_l1(v1, v2)\n",
    "    l2 = sim_l2(v1, v2)\n",
    "    cos = sim_cos(v1, v2)\n",
    "    vec = concatenate([Lambda(lambda x: K.reshape(x, shape=(-1, 1)))(t) for t in [l1, l2, cos]], axis=1)\n",
    "    return vec\n",
    "\n",
    "def similarity_mpcnn(s1, s2):\n",
    "    fea_h, fea_a = [], []\n",
    "    out1, out2 = [], []        \n",
    "    for i in range(len(s1)):\n",
    "        avg1 = GlobalAveragePooling1D()(s1[i])\n",
    "        max1 = GlobalMaxPooling1D()(s1[i])\n",
    "        min1 = Lambda(lambda x: K.min(x, axis=1))(s1[i])\n",
    "        out1.append([avg1, max1, min1])\n",
    "        \n",
    "        avg2 = GlobalAveragePooling1D()(s2[i])\n",
    "        max2 = GlobalMaxPooling1D()(s2[i])\n",
    "        min2 = Lambda(lambda x: K.min(x, axis=1))(s2[i])\n",
    "        out2.append([avg2, max2, min2])\n",
    "        \n",
    "    output1, output2 = [], [] # pool nums\n",
    "    for p in range(3):\n",
    "        output1.append(concatenate([Lambda(lambda x:K.reshape(x, shape=(-1, 1, CONV_LEN1)))(t[p]) for t in out1], axis=1))\n",
    "        output2.append(concatenate([Lambda(lambda x:K.reshape(x, shape=(-1, 1, CONV_LEN1)))(t[p]) for t in out2], axis=1))\n",
    "    \n",
    "    for p in range(3):\n",
    "        for f in range(CONV_LEN1):\n",
    "            fea_h.append(sim_vec(Lambda(lambda x: x[:, :, f])(output1[p]), Lambda(lambda x: x[:, :, f])(output2[p])))\n",
    "    \n",
    "    for p in range(3):\n",
    "        for k1 in range(len(s1)):\n",
    "            for k2 in range(len(s1)):\n",
    "                fea_a.append(sim_vec(Lambda(lambda x: x[:, k1, :])(output1[p]), Lambda(lambda x: x[:, k2, :])(output2[p])))\n",
    "    fea = concatenate(fea_h + fea_a, axis=1)\n",
    "    return fea"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# WORDS"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from sklearn.model_selection import StratifiedKFold\n",
    "\n",
    "best_results = []\n",
    "last_results = []\n",
    "best_file_names = []\n",
    "\n",
    "for i, (train_index, dev_index) in enumerate(StratifiedKFold(n_splits=10).split(X=word1, y=label)):  # word/char switch\n",
    "    print(\"fold {} start\".format(i + 1))\n",
    "    train_x1, train_x2, train_y = word1[train_index, :], word2[train_index, :], label[train_index]  # word/char switch\n",
    "    dev_x1, dev_x2, dev_y = word1[dev_index, :], word2[dev_index, :], label[dev_index]  # word/char switch\n",
    "    \n",
    "    input1 = Input(shape=(SEQ_LEN,), dtype=\"int32\")\n",
    "    input2 = Input(shape=(SEQ_LEN,), dtype=\"int32\")\n",
    "\n",
    "    embedding_layer = Embedding(\n",
    "        input_dim=word_embedding_data.shape[0],  # word/char switch\n",
    "        output_dim=word_embedding_data.shape[1],  # word/char switch\n",
    "        weights=[word_embedding_data],  # word/char switch\n",
    "        input_length=SEQ_LEN,\n",
    "        trainable=False\n",
    "    )\n",
    "    \n",
    "    vector1 = embedding_layer(input1)\n",
    "    vector2 = embedding_layer(input2)\n",
    "    \n",
    "    input_layer = TimeDistributed(Dense(DENSE_INPUT))\n",
    "    vector1 = input_layer(vector1)\n",
    "    vector1 = BatchNormalization()(vector1)\n",
    "    vector2 = input_layer(vector2)\n",
    "    vector2 = BatchNormalization()(vector2)\n",
    "    \n",
    "    lstm_layer1 = LSTM(LSTM_SIZE1, dropout=LSTM_DROP_RATE, recurrent_dropout=LSTM_DROP_RATE, return_sequences=True)\n",
    "    layer1a = lstm_layer1(vector1)\n",
    "    layer1a = Dropout(LSTM_DROP_RATE)(layer1a)\n",
    "    layer1b = lstm_layer1(vector2)\n",
    "    layer1b = Dropout(LSTM_DROP_RATE)(layer1b)\n",
    "    lstm_layer2 = LSTM(LSTM_SIZE2, dropout=LSTM_DROP_RATE, recurrent_dropout=LSTM_DROP_RATE, return_sequences=True)\n",
    "    layer2a = lstm_layer2(layer1a)\n",
    "    layer2b = lstm_layer2(layer1b)\n",
    "    \n",
    "    conv1a, conv1b = cnn_double_layer(layer2a, layer2b, filters=CONV_LEN1, kernel_size=1)\n",
    "    conv2a, conv2b = cnn_double_layer(layer2a, layer2b, filters=CONV_LEN2, kernel_size=2)\n",
    "    conv3a, conv3b = cnn_double_layer(layer2a, layer2b, filters=CONV_LEN3, kernel_size=3)\n",
    "    conv4a, conv4b = cnn_double_layer(layer2a, layer2b, filters=CONV_LEN4, kernel_size=4)\n",
    "    conv5a, conv5b = cnn_double_layer(layer2a, layer2b, filters=CONV_LEN5, kernel_size=5)\n",
    "    conv6a, conv6b = cnn_double_layer(layer2a, layer2b, filters=CONV_LEN6, kernel_size=6)\n",
    "\n",
    "    merge_a = concatenate([conv1a, conv2a, conv3a, conv4a, conv5a, conv6a])\n",
    "    merge_b = concatenate([conv1b, conv2b, conv3b, conv4b, conv5b, conv6b])\n",
    "    diff = Lambda(lambda x: K.abs(x[0] - x[1]))([merge_a, merge_b])\n",
    "    mult = Lambda(lambda x: x[0] * x[1])([merge_a, merge_b])\n",
    "    merge = concatenate([diff, mult])\n",
    "    \n",
    "    x = Dropout(DROP_RATE)(merge)\n",
    "    x = BatchNormalization()(x)\n",
    "    x = Dense(DENSE_SIZE1, activation=\"relu\")(x)\n",
    "    x = Dropout(DROP_RATE)(x)\n",
    "    x = BatchNormalization()(x)\n",
    "    x = Dense(DENSE_SIZE2, activation=\"relu\")(x)\n",
    "    x = Dropout(DROP_RATE)(x)\n",
    "    x = BatchNormalization()(x)\n",
    "    pred = Dense(1, activation=\"sigmoid\")(x)\n",
    "    \n",
    "    model = Model(inputs=[input1, input2], outputs=pred)\n",
    "    model.compile(\n",
    "        optimizer=\"nadam\",\n",
    "        loss=\"binary_crossentropy\",\n",
    "        metrics=[\"acc\"]\n",
    "    )\n",
    "    \n",
    "    early_stopping = EarlyStopping(\"val_loss\", patience=8)\n",
    "    lr_reducer = ReduceLROnPlateau(factor=0.5, patience=3, min_lr=0.001)\n",
    "    check_point = ModelCheckpoint(\n",
    "        \"./log/%s.Multi_LSTM_CNN_v4.word.{epoch:03d}.hdf5\" % (datetime.now().strftime(\"%Y%m%d-%H%M%S\")),  # word/char switch\n",
    "        monitor=\"val_loss\",\n",
    "        save_best_only=True,\n",
    "    )\n",
    "    \n",
    "    fit_res = model.fit(\n",
    "        x=[train_x1, train_x2],\n",
    "        y=train_y,\n",
    "        batch_size=BATCH_SIZE,\n",
    "        epochs=NUM_EPOCHES,\n",
    "        validation_data=([dev_x1, dev_x2], dev_y),\n",
    "        shuffle=True,\n",
    "        callbacks=[early_stopping, lr_reducer, check_point]\n",
    "    )\n",
    "    \n",
    "    pred_last = model.predict([test_word1, test_word2], batch_size=BATCH_SIZE)  # word/char switch\n",
    "    last_results.append(pd.DataFrame(pred_last, columns=[\"y_pre\"]))\n",
    "    \n",
    "    best_model_file = glob(\"./log/*.hdf5\")[-1].replace(\"\\\\\", \"/\")\n",
    "    best_file_names.append(best_model_file)\n",
    "    print(\"load model %s\" % (best_model_file,))\n",
    "    model.load_weights(best_model_file)\n",
    "    pred_best = model.predict([test_word1, test_word2], batch_size=BATCH_SIZE)  # word/char switch\n",
    "    best_results.append(pd.DataFrame(pred_best, columns=[\"y_pre\"]))\n",
    "\n",
    "pd.DataFrame(pd.concat(last_results, axis=1).mean(axis=1), columns=[\"y_pre\"]).to_csv(\n",
    "    \"./result/%s-Multi_LSTM_CNN_v4_word_last.csv\" % (datetime.now().strftime(\"%Y%m%d-%H%M%S\")),  # word/char switch\n",
    "    index=False\n",
    ")\n",
    "pd.DataFrame(pd.concat(best_results, axis=1).mean(axis=1), columns=[\"y_pre\"]).to_csv(\n",
    "    \"./result/%s-Multi_LSTM_CNN_v4_word_best.csv\" % (datetime.now().strftime(\"%Y%m%d-%H%M%S\")),  # word/char switch\n",
    "    index=False\n",
    ")\n",
    "\n",
    "model_path = \"./log/\" + datetime.now().strftime(\"%Y%m%d-%H%M%S\") + \"/\"\n",
    "os.mkdir(model_path)\n",
    "for model_name in best_file_names:\n",
    "    abs_name = os.path.split(model_name)[1]\n",
    "    os.rename(model_name, model_path + abs_name)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "collapsed": true
   },
   "source": [
    "# CHARS"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# from sklearn.model_selection import StratifiedKFold\n",
    "\n",
    "# best_results = []\n",
    "# last_results = []\n",
    "# best_file_names = []\n",
    "\n",
    "# for i, (train_index, dev_index) in enumerate(StratifiedKFold(n_splits=10).split(X=char1, y=label)):  # word/char switch\n",
    "#     print(\"fold {} start\".format(i + 1))\n",
    "#     train_x1, train_x2, train_y = char1[train_index, :], char2[train_index, :], label[train_index]  # word/char switch\n",
    "#     dev_x1, dev_x2, dev_y = char1[dev_index, :], char2[dev_index, :], label[dev_index]  # word/char switch\n",
    "    \n",
    "#     input1 = Input(shape=(SEQ_LEN,), dtype=\"int32\")\n",
    "#     input2 = Input(shape=(SEQ_LEN,), dtype=\"int32\")\n",
    "\n",
    "#     embedding_layer = Embedding(\n",
    "#         input_dim=char_embedding_data.shape[0],  # word/char switch\n",
    "#         output_dim=char_embedding_data.shape[1],  # word/char switch\n",
    "#         weights=[char_embedding_data],  # word/char switch\n",
    "#         input_length=SEQ_LEN,\n",
    "#         trainable=False\n",
    "#     )\n",
    "    \n",
    "#     vector1 = embedding_layer(input1)\n",
    "#     vector2 = embedding_layer(input2)\n",
    "    \n",
    "#     input_layer = TimeDistributed(Dense(DENSE_INPUT))\n",
    "#     vector1 = input_layer(vector1)\n",
    "#     vector1 = BatchNormalization()(vector1)\n",
    "#     vector2 = input_layer(vector2)\n",
    "#     vector2 = BatchNormalization()(vector2)\n",
    "    \n",
    "#     lstm_layer1 = LSTM(LSTM_SIZE1, dropout=LSTM_DROP_RATE, recurrent_dropout=LSTM_DROP_RATE, return_sequences=True)\n",
    "#     layer1a = lstm_layer1(vector1)\n",
    "#     layer1a = Dropout(LSTM_DROP_RATE)(layer1a)\n",
    "#     layer1b = lstm_layer1(vector2)\n",
    "#     layer1b = Dropout(LSTM_DROP_RATE)(layer1b)\n",
    "#     lstm_layer2 = LSTM(LSTM_SIZE2, dropout=LSTM_DROP_RATE, recurrent_dropout=LSTM_DROP_RATE, return_sequences=True)\n",
    "#     layer2a = lstm_layer2(layer1a)\n",
    "#     layer2b = lstm_layer2(layer1b)\n",
    "    \n",
    "#     conv1a, conv1b = cnn_double_layer(layer2a, layer2b, filters=CONV_LEN1, kernel_size=1)\n",
    "#     conv2a, conv2b = cnn_double_layer(layer2a, layer2b, filters=CONV_LEN2, kernel_size=2)\n",
    "#     conv3a, conv3b = cnn_double_layer(layer2a, layer2b, filters=CONV_LEN3, kernel_size=3)\n",
    "#     conv4a, conv4b = cnn_double_layer(layer2a, layer2b, filters=CONV_LEN4, kernel_size=4)\n",
    "#     conv5a, conv5b = cnn_double_layer(layer2a, layer2b, filters=CONV_LEN5, kernel_size=5)\n",
    "#     conv6a, conv6b = cnn_double_layer(layer2a, layer2b, filters=CONV_LEN6, kernel_size=6)\n",
    "\n",
    "#     merge_a = concatenate([conv1a, conv2a, conv3a, conv4a, conv5a, conv6a])\n",
    "#     merge_b = concatenate([conv1b, conv2b, conv3b, conv4b, conv5b, conv6b])\n",
    "#     diff = Lambda(lambda x: K.abs(x[0] - x[1]))([merge_a, merge_b])\n",
    "#     mult = Lambda(lambda x: x[0] * x[1])([merge_a, merge_b])\n",
    "#     merge = concatenate([diff, mult])\n",
    "    \n",
    "#     x = Dropout(DROP_RATE)(merge)\n",
    "#     x = BatchNormalization()(x)\n",
    "#     x = Dense(DENSE_SIZE1, activation=\"relu\")(x)\n",
    "#     x = Dropout(DROP_RATE)(x)\n",
    "#     x = BatchNormalization()(x)\n",
    "#     x = Dense(DENSE_SIZE2, activation=\"relu\")(x)\n",
    "#     x = Dropout(DROP_RATE)(x)\n",
    "#     x = BatchNormalization()(x)\n",
    "#     pred = Dense(1, activation=\"sigmoid\")(x)\n",
    "    \n",
    "#     model = Model(inputs=[input1, input2], outputs=pred)\n",
    "#     model.compile(\n",
    "#         optimizer=\"nadam\",\n",
    "#         loss=\"binary_crossentropy\",\n",
    "#         metrics=[\"acc\"]\n",
    "#     )\n",
    "    \n",
    "#     early_stopping = EarlyStopping(\"val_loss\", patience=8)\n",
    "#     lr_reducer = ReduceLROnPlateau(factor=0.5, patience=3, min_lr=0.001)\n",
    "#     check_point = ModelCheckpoint(\n",
    "#         \"./log/%s.Multi_LSTM_CNN_v4.char.{epoch:03d}.hdf5\" % (datetime.now().strftime(\"%Y%m%d-%H%M%S\")),  # word/char switch\n",
    "#         monitor=\"val_loss\",\n",
    "#         save_best_only=True,\n",
    "#     )\n",
    "    \n",
    "#     fit_res = model.fit(\n",
    "#         x=[train_x1, train_x2],\n",
    "#         y=train_y,\n",
    "#         batch_size=BATCH_SIZE,\n",
    "#         epochs=NUM_EPOCHES,\n",
    "#         validation_data=([dev_x1, dev_x2], dev_y),\n",
    "#         shuffle=True,\n",
    "#         callbacks=[early_stopping, lr_reducer, check_point]\n",
    "#     )\n",
    "    \n",
    "#     pred_last = model.predict([test_char1, test_char2], batch_size=BATCH_SIZE)  # word/char switch\n",
    "#     last_results.append(pd.DataFrame(pred_last, columns=[\"y_pre\"]))\n",
    "    \n",
    "#     best_model_file = glob(\"./log/*.hdf5\")[-1].replace(\"\\\\\", \"/\")\n",
    "#     best_file_names.append(best_model_file)\n",
    "#     print(\"load model %s\" % (best_model_file,))\n",
    "#     model.load_weights(best_model_file)\n",
    "#     pred_best = model.predict([test_char1, test_char2], batch_size=BATCH_SIZE)  # word/char switch\n",
    "#     best_results.append(pd.DataFrame(pred_best, columns=[\"y_pre\"]))\n",
    "\n",
    "# pd.DataFrame(pd.concat(last_results, axis=1).mean(axis=1), columns=[\"y_pre\"]).to_csv(\n",
    "#     \"./result/%s-Multi_LSTM_CNN_v4_char_last.csv\" % (datetime.now().strftime(\"%Y%m%d-%H%M%S\")),  # word/char switch\n",
    "#     index=False\n",
    "# )\n",
    "# pd.DataFrame(pd.concat(best_results, axis=1).mean(axis=1), columns=[\"y_pre\"]).to_csv(\n",
    "#     \"./result/%s-Multi_LSTM_CNN_v4_char_best.csv\" % (datetime.now().strftime(\"%Y%m%d-%H%M%S\")),  # word/char switch\n",
    "#     index=False\n",
    "# )\n",
    "\n",
    "# model_path = \"./log/\" + datetime.now().strftime(\"%Y%m%d-%H%M%S\") + \"/\"\n",
    "# os.mkdir(model_path)\n",
    "# for model_name in best_file_names:\n",
    "#     abs_name = os.path.split(model_name)[1]\n",
    "#     os.rename(model_name, model_path + abs_name)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
